<?php
require_once __DIR__.'/rssCollector.php';
class SinaRssCollector extends rssCollector{
    public $channels = array();
    //新闻来源
    public $rssSourceMap = array();
    //上次抓取时间
    public $lastGetTime = '';
    //本次成功抓取总数
    public $successCount = 0;
    //本次失败抓取总数
    public $errorCount = 0;
    //总时间
    public $totalTime = 0;
    //网络传输耗费时间
    public $transferTime = 0;
    //正则表达式解析时间
    public $parseTime = 0;
    //入库时间
    public $saveTime = 0;
    public $count1 = 0;
    public $count = 0;
    
    //构造函数，初始化资源列表和上次抓取时间
    public function __construct() {
		parent::__construct();
		$this->dataPath = __DIR__.'/sinaData.php';
        $this->rssSourceMap = $this->getCategoryMap();
        $this->lastGetTime = $this->getLastGetTime();
    }
    
    //执行抓取
    public function execut() {
        $beginTotalTime = microtime(true);
        //每个大类依次处理
        foreach ($this->rssSourceMap as $cname1 => $category){
            //获取次类列表,不包含详细内容，关键词等
            $listSimple = $this->getList($category, $cname1);
            //根据频道，使用相应的方式处理具体内容
            $listDetail = $this->getDetail($listSimple);
            $this->count1 = count($listDetail);
            //保存至数据库
            $beginSaveTime = microtime(true);
            foreach ($listDetail as $key=>$newsListArr){
                $this->saveToDB($newsListArr);
            }
            $endSaveTime = microtime(true);
            $this->saveTime = $endSaveTime - $beginSaveTime;
        }
        $endTotalTime = microtime(true);
        $this->totalTime = $endTotalTime - $beginTotalTime;
        $this->setLastGetTime();
    }
    
    //获取给定分类的简单列表
    private function getList($category, $cname1){
        $beginTransferTime = microtime(true);
        $nodes = array();
        foreach ($category as $key => $value){
            $nodes[$key] = 'http://rss.sina.com.cn'.$value;
        }
        $res = $this->multipleThreadsRequest($nodes);
        $endTransferTime = microtime(true);
        $this->transferTime += $endTransferTime - $beginTransferTime;
        //结果过滤+格式整理
        $beginParseTime = microtime(true);
        foreach ($res as $key => $newsListXML){
            //获取次类列表
            $newsListArrRaw = $this->convertXMLToArray($newsListXML);
            $newsListArr = $newsListArrRaw['channel']['item'];
            foreach ($newsListArr as $key1 => $newsSimple){
                //没有日期的直接放弃
                if(!$newsSimple['pubDate']){
                    unset($newsListArr[$key1]);
                    $this->errorCount += 1;
                    continue;
                }
                $newsPubTime = $newsListArr[$key1]['pubDate'] = strtotime($newsSimple['pubDate']);
                //忽略已经抓过的
                if($newsPubTime < $this->lastGetTime){
                    unset($newsListArr[$key1]);
                    continue;
                }
                //标记来源
                $newsListArr[$key1]['source'] = 'sina';
                //链接处理
                $newsListArr[$key1]['link'] = $newsLink = str_replace('http://go.rss.sina.com.cn/redirect.php?url=', '', $newsSimple['link']);
                //url为空的直接放弃
                if(!$newsLink){
                    unset($newsListArr[$key1]);
                    $this->errorCount += 1;
                    continue;
                }
                //匹配出频道
                preg_match("/ *http:\/\/([a-xA-Z.]*?)\.sina\.com\.cn\/.*/is", $newsLink, $matches);
                $newsListArr[$key1]['channel'] = isset($matches[1]) ? trim($matches[1]) : ''; 
                //分类
                $newsListArr[$key1]['category'] = $key.'-'.$cname1;
                //标题格式化
                $newsListArr[$key1]['title'] = preg_replace("/\s+/","",$newsSimple['title']);
                //简介格式化
                if(!$newsSimple['description'] || is_array($newsSimple['description'])){
                    $newsListArr[$key1]['description'] = $newsSimple['description'] = '';
                }
                $newsListArr[$key1]['description'] = preg_replace("/\s+/","",$newsSimple['description']);

                if(!in_array($newsListArr[$key1]['channel'], $this->channels)){
                    $this->channels[] = $newsListArr[$key1]['channel'];
                }
            }
            $res[$key] = $newsListArr;
        }
        $endParseTime = microtime(true);
        $this->parseTime += $endParseTime - $beginParseTime;
        return $res;
    }
    
    //获取详细内容
    private function getDetail($res){
        foreach ($res as $key => $newsListArr){
            $beginTransferTime = microtime(true);
            //组装抓取节点
            $nodes = array();
            foreach ($newsListArr as $key1 => $newsArr){
                $nodes[$key1] = $newsArr['link'];
            }
            //获取列表内容
            $newsListContent = $this->multipleThreadsRequest($nodes);
            $endTransferTime = microtime(true);
            $this->transferTime += $endTransferTime - $beginTransferTime;
            //匹配内容和关键词
            $beginParseTime = microtime(true);
            foreach ($newsListArr as $key2 => $value2){
                $this->successCount += 1;
                //网页编码解析
                //$charset = preg_match("/<meta.+?charset=[^\w]?([-\w]+)/i", $newsListContent[$key2], $temp) ? strtolower($temp[1]) : "gbk"; 
                $content = iconv('GBK', 'UTF-8//IGNORE', $newsListContent[$key2]);
                if(strpos($value2['channel'], 'slide') !== false){
                    $newsListArr[$key2]['slides'] = $this->matchSlide($content);
                    $newsListArr[$key2]['video'] = '';
                    $newsListArr[$key2]['content'] = '';
                    $newsListArr[$key2]['keywords'] = '';
                    continue;
                }
                if($value2['channel'] == 'video'){
                    $newsListArr[$key2]['video'] = $this->matchVideo($content);
                    $newsListArr[$key2]['slides'] = '';
                    $newsListArr[$key2]['content'] = '';
                    $newsListArr[$key2]['keywords'] = '';
                    continue;
                }
                $newsListArr[$key2]['slides'] = '';
                $newsListArr[$key2]['video'] = '';
                $newsListArr[$key2]['content'] = $this->matchContnet($content);
                $newsListArr[$key2]['keywords'] = $this->matchKeywords($content); 
            }
            $endParseTime = microtime(true);
            $this->parseTime += $endParseTime - $beginParseTime;
            $res[$key] = $newsListArr;
        }
        return $res;
    }
    
    //匹配内容
    private function matchContnet($content){
        $matches = array();
        preg_match('/<!-- publish_helper(?:.*?)-->(.*?)<!-- publish_helper_end -->/is', $content, $matches);
        return isset($matches[1]) ? trim($matches[1]) : '';
    }
    
    //匹配关键词
    private function matchKeywords($content){
        $matches = array();
        preg_match('/<p class="art_keywords"(.*?)>(.*?)<\/p>/is', $content, $matches);
        $resHtml = isset($matches[2]) ? trim($matches[2]) : '';
        preg_match_all("/<a href=(?:.*?)>(.*?)<\/a>/is", $resHtml, $matches);
        $keywordsArr = isset($matches[1]) ? $matches[1] : array();
        return $keywordsArr ? implode(',', $keywordsArr) : '';
    }
    
    //匹配slide图片内容
    private function matchSlide($content){
        $matches = array();
        //匹配出数据区域
        preg_match('/<div id="eData" style="display:none;">(.*?)<\/div>/is', $content, $matches);
        $resHtml = isset($matches[1]) ? trim($matches[1]) : '';
        //匹配出数据条目
        preg_match_all('/<dl>(.*?)<\/dl>/is', $resHtml, $matches);
        $items = isset($matches[1]) ? $matches[1] : array();
        //组合结果
        $res = array();
        foreach ($items as $key => $value){
            preg_match('/<dt>(.*?)<\/dt>/is', $value, $matches);
            $res[$key]['title'] = $matches[1];
            preg_match_all('/<dd>(.*?)<\/dd>/is', $value, $matches);
            $res[$key]['url'] = $matches[1][0];
            $res[$key]['description'] = $matches[1][4];
        }
        return json_encode($res);
    }
    
    //匹配出视频内容
    public function matchVideo($content){
        $res = array();
        //匹配出视频基本信息
        $matches = array();
        preg_match('/title:\'(.*?)\'(?:.*)swfOutsideUrl:\'(.*?)\'/is', $content, $matches);
        $res['title'] = isset($matches[1]) ? trim($matches[1]) : '';
        $res['url'] = isset($matches[2]) ? trim($matches[2]) : '';
        //获取视频简介
        preg_match('/<em task="oldinfor" style="display:none;">(.*?)<\/em>/is', $content, $matches);
        $res['description'] = isset($matches[1]) ? trim($matches[1]) : '';
        //获取视频标签
        preg_match('/<p class="tags">(.*?)<\/p>/is', $content, $matches);
        $labelHtml = isset($matches[1]) ? trim($matches[1]) : '';
        preg_match_all("/<a href=(?:.*?)>(.*?)<\/a>/is", $labelHtml, $matches);
        $labelArr = isset($matches[1]) ? $matches[1] : array();
        $res['label'] = $labelArr ? implode(',', $labelArr) : '';
        return json_encode($res);
    }
    
    //保存至数据库
    protected function saveToDB($newsArr){
        try {
            foreach ($newsArr as $news){
                $this->count += 1;
                $sql = "INSERT INTO news1(source, source_url, category, channel, author, title, keywords, description, content, slides, video, publishTime) VALUES(:source, :source_url, :category, :channel, :author, :title, :keywords, :description, :content, :slides, :video, :publishTime)";
                $data['source'] = 'sina';
                $data['source_url'] = $news['link'];
                $data['category'] = $news['category'];
                $data['channel'] = $news['channel'];
                $data['author'] = $news['author'];
                $data['title'] = $news['title'];
                $data['keywords'] = $news['keywords'];
                $data['description'] = $news['description'];
                $data['content'] = $news['content'];
                $data['slides'] = $news['slides'];
                $data['video'] = $news['video'];
                $data['publishTime'] = $news['pubDate'];
                $this->dbCon->prepare($sql)->execute($data);
            }
        } catch (PDOException $ex) {
            exit('error');
        }
    }
    
    //获取上次抓取时间
    private function getLastGetTime(){
		$data = require $this->dataPath;
		return $data['last_get_time'];
    }
    
    //设置上次抓取时间为现在时间
    private function setLastGetTime(){
		$data = require $this->dataPath;
		$data['last_get_time'] = time();
		$data_str = '<?php return '.var_export($data, true).';';
        file_put_contents($this->dataPath, $data_str);
    }
    
    //获取RSS源
    public function getCategoryMap(){
        return array(
            '新闻中心' => array(
                '新闻要闻' => '/news/marquee/ddt.xml',
                '国内要闻' => '/news/china/focus15.xml',
                '国际要闻' => '/news/world/focus15.xml',
                '社会新闻' => '/news/society/focus15.xml',
                '港澳台新闻' => '/news/china/hktaiwan15.xml',
                '社会与法' => '/news/society/law15.xml',
                '社会万象' => '/news/society/misc15.xml',
                '真情时刻' => '/news/society/feeling15.xml',
                '奇闻轶事' => '/news/society/wonder15.xml',
            ),
            '体育新闻' => array(
                '体育要闻汇总' => '/roll/sports/hot_roll.xml',
                '焦点新闻' => '/news/allnews/sports.xml',
                '国际足坛' => '/sports/global/focus.xml',
                '意甲' => '/sports/global/italy.xml',
                '英超' => '/sports/global/england.xml',
                '西甲' => '/sports/global/spain.xml',
                '德甲、法甲' => '/sports/global/germanyfrance.xml',
                '五洲热报' => '/sports/global/others.xml',
                '精彩图片' => '/sports/global/photo.xml',
                '国内足坛' => '/sports/china/focus.xml',
                '中国之队' => '/sports/china/team.xml',
                '篮球新闻' => '/sports/basketball/focus.xml',
                'NBA新闻' => '/sports/basketball/nba.xml',
                'CBA新闻' => '/sports/basketball/cba.xml',
                '综合体育' => '/sports/others/focus.xml',
                '彩票新闻' => '/sports/global/lottery.xml',
                '高尔夫新闻' => '/sports/global/golf.xml',
                'F1赛车新闻' => '/sports/global/f1.xml',
                '棋牌新闻' => '/sports/global/chess.xml',
            ),
            '科技新闻' => array(
                '科技要闻汇总' => '/tech/rollnews.xml',
                '焦点新闻' => '/news/allnews/tech.xml',
                '互联网新闻' => '/tech/internet/home28.xml',
                '手机资讯' => '/tech/mobile/mobile_6.xml',
                '4G新闻' => '/tech/4G.xml',
                '笔记本新闻' => '/tech/notebook/193_1.xml',
                '电信新闻' => '/tech/tele/gn37.xml',
                '业界新闻' => '/tech/it/gn37.xml',
                '科普要闻' => '/tech/discovery/discovery.xml',
                '数码资讯' => '/tech/number/new_camera.xml',
                '家电资讯' => '/tech/elec/buy_elec.xml',
            ),
            '财经新闻' => array(
                '财经要闻汇总' => '/roll/finance/hot_roll.xml',
                '焦点新闻' => '/news/allnews/finance.xml',
                '股市及时雨' => '/finance/jsy.xml',
                '股票要闻汇总' => '/roll/stock/hot_roll.xml',
                '基金要闻' => '/finance/fund.xml',
                '理财要闻' => '/finance/financial.xml',
                '美股快报' => '/finance/usstock.xml',
                '港股快讯' => '/finance/hkstock.xml',
                '期货要闻' => '/finance/future.xml',
            ),
            '军事新闻' => array(
                '军事要闻汇总' => '/roll/mil/hot_roll.xml',
                '焦点新闻' => '/jczs/focus.xml',
                '国际军情' => '/jczs/taiwan20.xml',
                '中国军情' => '/jczs/china15.xml',
            ),
            /*'女性新闻' => array(
                '女性要闻汇总' => '/news/allnews/eladies.xml',
                '情感婚姻' => '/eladies/marry.xml',
                '八卦' => '/eladies/gossip.xml',
                '亲子' => '/eladies/son.xml',
            ),*/
            '汽车新闻' => array(
                '焦点新闻' => '/news/allnews/auto.xml',
                '行业动态' => '/auto/news/t/index.xml',
                '购车指导' => '/auto/guide/index.xml',
                '新车上市' => '/auto/newcar/index.xml',
                '车市·降价' => '/auto/sales/index.xml',
                '企业新闻' => '/auto/news/e/index.xml',
                '汽车养护' => '/auto/servicing/index.xml',
                '汽车召回' => '/auto/recalls/index.xml',
                '汽车保险' => '/auto/insurance/index.xml',
                '汽车用品' => '/auto/automotive/index.xml',
                '改装' => '/auto/ultratune/index.xml',
                '交通' => '/auto/traffic/index.xml',
                '车致' => '/auto/um/index.xml',
            ),
            '娱乐新闻' => array(
                '娱乐要闻汇总' => '/ent/hot_roll.xml',
                '焦点新闻' => '/news/allnews/ent.xml',
                '音乐' => '/ent/music/focus12.xml',
                '电视前沿' => '/ent/tv/focus7.xml',
                '电影宝库' => '/ent/film/focus7.xml',
                '明星全接触' => '/ent/star/focus7.xml',
                '内地明星' => '/ent/inner.xml',
                '港台明星' => '/ent/hongkong.xml',
                '日韩明星' => '/ent/star/japan.xml',
                '欧美明星' => '/ent/star/outer.xml',   
            ),	
            '文化教育' => array(
                '教育要闻汇总' => '/roll/edu/hot_roll.xml',
                '焦点新闻' => '/edu/focus19.xml',
                '考试频道' => '/edu/exam.xml',
                '高考' => '/edu/gaokao.xml',
                '外语频道' => '/edu/en.xml',
                '考研' => '/edu/kaoyan.xml',
                '中考' => '/edu/zhongkao.xml',
                '公务员' => '/edu/gwy.xml',
                '英语考试' => '/edu/yyks.xml',
                '司法考试' => '/edu/sifa.xml',
                '会计考试' => '/edu/cpa.xml',
                '出国' => '/edu/abroad5.xml',
                '中小学教育' => '/edu/zxx.xml',
                '商学院' => '/edu/bschool.xml',
            ),
            '视频新闻' => array(
                '新闻视频排行' => '/bn/hot_bn.xml',
                '新闻' => '/bn/news.xml',
                '科技' => '/bn/tech.xml',
                '财经' => '/bn/finance.xml',
                '体育' => '/bn/sports.xml',
                '娱乐' => '/bn/ent.xml',
                '时尚' => '/bn/fashion.xml',
            ),
            '育儿指导' => array(
                '育儿知识' => '/baby/news/guide.xml',
                '育儿新闻' => '/baby/news/news.xml',
                '准备怀孕' => '/baby/news/zhunbei.xml',
                '怀孕期' => '/baby/news/huaiyun.xml',
                '分娩期' => '/baby/news/fenmian.xml',
                '新生儿' => '/baby/news/xinsheng.xml',
                '婴儿期' => '/baby/news/yinger.xml',
                '幼儿期' => '/baby/news/youer.xml',
                '学龄前' => '/baby/news/xueqian.xml',
            ),
        );
    }
}